package SparkSQL2

import org.apache.spark.SparkConf
import org.apache.spark.sql.{Dataset, KeyValueGroupedDataset, SparkSession}

object DatasetWordCount {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    val sparkSession = SparkSession
      .builder()
      .master("local")
      .config(conf)
      .appName(s"${this.getClass.getSimpleName}")
      .getOrCreate()

    import sparkSession.implicits._
    val data = sparkSession.read.text("/Users/hongyi/IdeaProjects/SparkSQLTest/src/main/scala/SparkSQL2/hello.txt").as[String]

    val words: Dataset[String] = data.flatMap(value=>value.split(","))
    val groupedWords: KeyValueGroupedDataset[String, String] = words.groupByKey(_.toLowerCase())
    val counts: Dataset[(String, Long)] = groupedWords.count()
    counts.show()

  }
}
